import requests, re, random, pymongo
from urllib.request import quote
from pyquery import PyQuery as pq
work_urls = set()
proxy_list = []


def get_proxies():
    """随机获取代理IP"""
    url = "https://www.kuaidaili.com/free/"
    req = requests.get(url)
    html = req.content.decode("utf-8")
    doc = pq(html)
    tr_all = doc("tr").items()
    for proxy in tr_all:
        ip = proxy.find("[data-title='IP']").text()
        port = proxy.find("[data-title='PORT']").text()
        if ip:
            proxy_list.append(ip + ":" + port)
    proxy = random.choice(proxy_list)
    print("当前代理IP为：" + proxy)
    proxies = {
        'http': 'http://' + proxy,
        'https': 'https://' + proxy,
    }
    return proxies


def get_work_url(page, key, proxies):
    url = "http://zhaogong.chinalao.com/k-" + key + "/" + str(page) + "/"
    req = requests.get(url, """proxies=proxies""")
    html = pq(req.content.decode('utf-8'))
    items = html("h2.wtit a").items()
    for i in items:
        work_urls.add(i.attr("href"))
    return work_urls


def parse(proxies):
    while work_urls:
        work_url = work_urls.pop()
        req = requests.get(work_url, """proxies=proxies""")
        html = req.content.decode('utf-8')
        doc = pq(html)
        yield {
            'title': doc.find("h1.jobtitle").text(),
            'date': doc.find("p.jobdate").text(),
            'price': doc.find("p.jobprice").text(),
            'work_add': ''.join(re.findall('<li><span>工作地点：</span>(.*?)<address', html)).strip(),
            'sex_limit': ''.join(re.findall('<li><span>性别要求：</span>(.*?)</li>', html)).strip(),
            'age_limit': ''.join(re.findall('<li><span>年龄要求：</span>(.*?)</li>', html)).strip(),
            'person_count': doc.find('p.f_black span em').text()
        }


def save_in_mongo(job, job_info):
    job.insert_one(job_info)


def main():
    key = '普工'
    for page in range(1, 3):
        proxies = get_proxies()
        get_work_url(page, quote(key), proxies)
    client = pymongo.MongoClient("localhost", 27017)
    job = client.chinalao.job
    for item in parse(proxies):
        save_in_mongo(job, item)
    client.close()


if __name__ == '__main__':
    main()
